""" make_xml.py for mwe
 Converts extended ascii into &#xHHHH; form
 Converts x80...x9F from cp1252 to unicode form
 The text is almost all in HK transliteration, as in skd.txt.
 Conversion to slp will be done in the displays.
"""
import xml.etree.ElementTree as ET
import sys, re
import transcoder
transcoder.transcoder_set_dir("");
xmlroot = "mwe"
cp1252 = {  # see doubledecode.py
    # from http://www.microsoft.com/typography/unicode/1252.htm
    "&#x20AC;": "\x80", # EURO SIGN
    "&#x201A;": "\x82", # SINGLE LOW-9 QUOTATION MARK
    "&#x0192;": "\x83", # LATIN SMALL LETTER F WITH HOOK
    "&#x201E;": "\x84", # DOUBLE LOW-9 QUOTATION MARK
    "&#x2026;": "\x85", # HORIZONTAL ELLIPSIS
    "&#x2020;": "\x86", # DAGGER
    "&#x2021;": "\x87", # DOUBLE DAGGER
    "&#x02C6;": "\x88", # MODIFIER LETTER CIRCUMFLEX ACCENT
    "&#x2030;": "\x89", # PER MILLE SIGN
    "&#x0160;": "\x8A", # LATIN CAPITAL LETTER S WITH CARON
    "&#x2039;": "\x8B", # SINGLE LEFT-POINTING ANGLE QUOTATION MARK
    "&#x0152;": "\x8C", # LATIN CAPITAL LIGATURE OE
    "&#x017D;": "\x8E", # LATIN CAPITAL LETTER Z WITH CARON
    "&#x2018;": "\x91", # LEFT SINGLE QUOTATION MARK
    "&#x2019;": "\x92", # RIGHT SINGLE QUOTATION MARK
    "&#x201C;": "\x93", # LEFT DOUBLE QUOTATION MARK
    "&#x201D;": "\x94", # RIGHT DOUBLE QUOTATION MARK
    "&#x2022;": "\x95", # BULLET
    "&#x2013;": "\x96", # EN DASH
    "&#x2014;": "\x97", # EM DASH
    "&#x02DC;": "\x98", # SMALL TILDE
    "&#x2122;": "\x99", # TRADE MARK SIGN
    "&#x0161;": "\x9A", # LATIN SMALL LETTER S WITH CARON
    "&#x203A;": "\x9B", # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
    "&#x0153;": "\x9C", # LATIN SMALL LIGATURE OE
    "&#x017E;": "\x9E", # LATIN SMALL LETTER Z WITH CARON
    "&#x0178;": "\x9F" # LATIN CAPITAL LETTER Y WITH DIAERESIS
}
cp1252_rev = {}
for k,v in cp1252.iteritems():
 cp1252_rev[v] = k
import headword
reHeadword0 = headword.reHeadword
reHeadword = re.compile(reHeadword0)
def adjust_ea(s):
 """ convert a string, changing extended ascii characters
     to &#xHHHH; form, taking into account cp1252 code conversions
 """
 out=[]
 for c in s:
  ic = ord(c)
  if (ic > 127):
   h = cp1252_rev.get(c)
   if (not h):
    h = "&#x%0.4X;" % ic
  else:
   h = c
  out.append(h)
 ans = ''.join(out)
 return ans

def adjust_tag(x,j):
 # j not used
 x = re.sub(r'<H>','<H/>',x)
 #x = re.sub(r'<>','',x) # <lb/> will be introduced later
 #x = re.sub(r'<HI>','<HI/>',x)
 x = re.sub(r'<HI>','',x)
 #x = re.sub(r'<C([0-9]+)>',r'<C n="\1"/>',x)
 #x = re.sub(r'<P>','<P/>',x)
 return x
def add_tag(x,j):
 x0 = x
 # add tags for {@@} = <b></b> 
 # {%%} = <i></i>
 # put bold indicator outside of sanskrit. This covers some situations,
 # perhaps not all.
 #x = re.sub(r'\{#\{@([^@]*?)@\}#\}',r'{@{#\1#}@}',x)
 x = re.sub(r'{%','<i>',x)
 x = re.sub(r'%}','</i>',x)
 x = re.sub(r'{#','<s>',x)
 x = re.sub(r'#}','</s>',x) 
 return x
def adjust_xml(line):
 line = re.sub(r'&','&amp;',line)
 line = re.sub(r' < ',' &lt; ',line)
 #line = re.sub(r' <$',' &lt;',line)
 return line

def adjust_key2(x):
 # Does nothing for mwe
 hw = x
 return hw
 m1 = re.search(r'^(.*?),',hw)
 if m1:
  hw = m1.group(1)
 else:
  m2 = re.search(r'^(.*?) ',hw)
  if m2:
   hw = m2.group(1)
 hwslp = re.sub(r'^(.*)$',adjust_hk,hw)
 # remove the <s> elements inserted by adjust_hk
 hwslp = re.sub(r'<s>','',hwslp)
 hwslp = re.sub(r'</s>','',hwslp)

 return hwslp

def old_adjust_hk(m):
 x = m.group(1)
 # This logic assumes, with dtd, that only <br/> element may be present.
 parts = re.split('<br/>',x)
 parts1 = []
 for part in parts:
  y = transcoder.transcoder_processString(part,'hk','slp1')
  parts1.append(y)
 ans = '<br/>'.join(parts1)
 return "<s>%s</s>" % ans

def adjust_hk(m):
 x = m.group(1)
 outarr = []
 parts = re.split(r'(<[^>]+>)|(\[Page.*?\])|(&.*?;)',x) # xml tags
 for part in parts: 
  if not part: #why needed? 
   pass 
  elif part.startswith('<') and part.endswith('>'):
   outarr.append(part)
  elif part.startswith('[Page') and part.endswith(']'):
   outarr.append(part)
  elif part.startswith('&') and part.endswith(';'):
   outarr.append(part)
  else: # assume text in hk. Convert to slp1
   #z = re.sub(r'\|','.',part) # text has non-standard | for danda
   #z = re.sub(r'MM','M',z) # text has MM for chandra bindu
   #z = re.sub(r'D2','D',z) # retroflex 'd' with dot under
   z = part
   y = transcoder.transcoder_processString(z,'hk','slp1')
   outarr.append("<s>%s</s>" % y)
 ans = ''.join(outarr)
 return ans
def construct_data(datalines,key1,lnum,page,col,n1):
 # construct data analogous to the way it is in mw
 # replace extended ascii in all lines
 datalines1 = []
 for line in datalines:
  line = line.strip()
  # change '..' and '...' to horizontal ellipsis
  #line = re.sub(r'[.][.]+',r'\x85',line)
  #line = re.sub(r'\x85',' ',line) # HORIZONTAL ELLIPSIS
  line = adjust_xml(line)
  line = adjust_ea(line)
  datalines1.append(line)
 # 1. h (head)
 line = datalines1[0].strip()
 m = reHeadword.search(line)
 if m:
  key2 = m.group(1)
  key2 = adjust_key2(key2)
 else :
  key2 = key1
 # force key2=key1 for mwe
 key2 = key1
 h = "<key1>%s</key1><key2>%s</key2>" % (key1,key2)
 #2. construct tail
 #ipage = int(page)
 ipage = page
 tail = "<L>%s</L><pc>%s</pc>" % (lnum,ipage)
 #3. construct body
 bodylines = []
 j = n1
 for x in datalines1:
  x = x.strip()
  #if re.search(r'<H>',x):
  # continue
  x = adjust_tag(x,j) # j not used here
  x = add_tag(x,j)
  #x = re.sub(r'^(.*?)$',adjust_hk,x)
  j = j + 1
  bodylines.append(x)
 body = '<lb/>'.join(bodylines)
 # convert <s>X</s> to slp1
 body = re.sub(r'<s>(.*?)</s>',adjust_hk,body)
 #if key1 == 'doll':
 #  print "dbg:\n before hk = \n%s\nafter hk = \n%s\n" % (body,x1)
 #body = x1
 #4. construct result
 data = "<H1><h>%s</h><body>%s</body><tail>%s</tail></H1>" % (h,body,tail)
 return data

def make_xmlfun(filein,filein1,fileout):
 # slurp txt file into list of lines
 with open(filein) as f:
    inlines = f.readlines()
 # open output xml file, and write header
 fout = open(fileout,'w')
 # write header lines
# write the preface pages
 text = """
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE %s SYSTEM "%s.dtd">
<!-- Copyright Universitat Koln 2013 -->
<%s>
""" % (xmlroot,xmlroot,xmlroot)
 lines = text.splitlines()
 nout = 0  # count of lines written
 for line in lines:
  line = line.strip()
  if (line != ''):
   fout.write("%s\n" %line)
   nout = nout + 1
 # read headword lines, and generate output
 f = open(filein1,'r')
 n = 0 # count of lines read
 lnum = 0 # generate record number for xml records constructed
 for line in f:
  n = n+1
  line = line.strip() # remove starting or ending whitespace
  (page,hwslp,linenum1,linenum2) = re.split('[:,]',line)
  col = 1 # there is no column for ap. 
  n1 = int(linenum1) - 1 # make 0-based
  n2 = int(linenum2) - 1
  datalines = inlines[n1:n2+1]
  # construct output
  lnum = lnum + 1
  key1 = hwslp
  data = construct_data(datalines,key1,lnum,page,col,n1)
  out = data
  fout.write("%s\n" % out)
  # check that out is well-formed xml
  try:
   root = ET.fromstring(out)
  except:
   print "xml error: n=%s,line=%s" %(n,line)
   print out,"\n"
  # for debug
  if (n > 100000):
   print "stopping after %s lines"%n
   break
  #if (key1 == 'aSva'):
  # print "stopping at aSva"
  # break
 # write closing line
 fout.write("</%s>\n" % xmlroot)
 fout.close()

if __name__=="__main__":
 filein = sys.argv[1] # acc.txt
 filein1 = sys.argv[2] #acchw2.txt
 fileout = sys.argv[3] # acc.xml
 make_xmlfun(filein,filein1,fileout)
